import requests, os
from bs4 import BeautifulSoup

RES_TXT = "./tmp/res.txt"
CLEAR_RES_TXT = "./tmp/clear_res.txt"
ZH_TXT = "./tmp/zh.txt"
EN_TXT = "./tmp/en.txt"
ENCODING = "utf-8-sig"

'''
    :param filename:html文件名
    :return:去除标签之后的网页文本
'''


def crawl_all_text(dirname: str):
    file_txt_list = []
    res_txt_list = []
    filenames = os.listdir(dirname)
    for file in filenames:
        if file[-4:] == "html":
            crud_file = open(dirname + "/" + file, "r", encoding=ENCODING)
            soup = BeautifulSoup(crud_file.read(), "lxml")
            file_txt_list.append(soup.get_text().strip())
            crud_file.close()
    for i in file_txt_list:
        res_txt_list.append(i.split("\n"))
    return res_txt_list

'''
    清除文本中的空行、换行符、以及神秘的字符
'''


def clear_txt(text_list):
    clear_list = []
    i = 0
    for text in text_list:
        for line in text:
            line = line.split()
            if line == []:
                continue
            else:
                line = " ".join(line)
                line += "\n"
                line = line.encode("utf-8").decode(ENCODING)
                if line != "\n":
                    clear_list.append(line)
    return clear_list

'''
    :param word:需要判断的句子
    判断句子中是否有中文
'''


def isChinese(word: str):
    for ch in word:
        if u'\u4e00' <= ch <= u'\u9fff':
            return True
    return False


'''
    :param word:需要判断的句子
    判断句子中是否有英文
'''


def isEnglish(word: str):
    for ch in word:
        if 65 <= ord(ch) <= 90 or 97 <= ord(ch) <= 122:
            return True
    return False


def convert_zh_en(clear_list):
    zh_list, en_list = [], []
    for line in clear_list:
        zh_flag, en_flag = 0, 0
        if isEnglish(line):
            en_flag = 1
        if isChinese(line):
            zh_flag = 1
        if en_flag == 1 and zh_flag == 0:
            if len(line.split(" ")) != 1:
                en_list.append(line.strip("\n"))
        if en_flag == 0 and zh_flag == 1:
            if len(line) > 3:
                zh_list.append(line.strip("\n"))
    return zh_list, en_list

